{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.image as mpimg\n",
    "import seaborn as sns\n",
    "%matplotlib inline\n",
    "\n",
    "np.random.seed(2)\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import itertools\n",
    "\n",
    "from keras.utils.np_utils import to_categorical # convert to one-hot-encoding\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D\n",
    "from keras.optimizers import RMSprop\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.callbacks import ReduceLROnPlateau\n",
    "\n",
    "\n",
    "sns.set(style='white', context='notebook', palette='deep')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Data preparation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.1 Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('./digit-recognizer/train.csv')\n",
    "test = pd.read_csv('./digit-recognizer/test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((42000, 784), (42000,))"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAEJCAYAAABohnsfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAATw0lEQVR4nO3de5BdVZXH8W/S3dEMCEgFTGJAoZCFOgNBecwMr6gghaKIIyJEICgghVjUiIJKGB4lzgylQXGIUvIIThCYAXEsIb6i8hgEH7xKkTXMqNFIKFRUXgbSSeaPc9pcQnezSfU596b7+6lK0Xfdc7MXSad/d599zz6T1q5diyRJJSZ3uwFJ0sbD0JAkFTM0JEnFDA1JUjFDQ5JUrL/bDTQlIl4A7A6sAFZ3uR1J2lj0ATOAH2bmU+s/OW5Dgyowbul2E5K0kdoHuHX94ngOjRUAV155JdOnT+92L5K0UXjooYeYO3cu1D9D1zeeQ2M1wPTp05k1a1a3e5Gkjc2wp/VdCJckFTM0JEnFDA1JUjFDQ5JUzNCQJBUzNCRJxQwNSVIxQ6NlawZXjcuxJE0M4/nivp40uX+AH59/XCtjvfa0S1oZR9LE4UxDklTM0JAkFTM0JEnFDA1JUjFDQ5JUzNCQJBUzNCRJxQwNSVIxQ0OSVMzQkCQVMzQkScUMDUk9ZXBwcFyONV64YaGkntLf38+nPvWpVsY69dRTWxlnPHGmoa5Z/XR7W7e3OZY0njnTUNf0TRngxqOPbWWsN33x8lbGkcY7ZxqSpGKGhiSpmKEhSSpmaEiSihkakqRihoYk9ahVq9f03Fh+5HaCenpwFVP6B8bNONJ4NNA3mQ9ef1MrYy04dL+i4wyNCWpK/wDzLj+l8XEWHfuZxsfQ2FkzuJrJ/X3jZhyNvQkVGk+vWs2Ugea/UdsaRxprk/v7uGfh9xofZ5eT5jQ+hpoxoUJjykAfR552ZePjfOn8uY2PIUnd0HhoRMQngWmZOS8iZgOXAJsBNwMnZuZgRGwLLAa2BhKYm5mPR8QWwJXA9sBvgXdm5kNN96yJZXDVavpbmBm2NY7UpEZDIyLeABwD3FCXFgPHZebtEXEpcDzwOWAhsDAzr46IM4EzgdOBjwO3ZOabI+Io4DPA4U32rImnf6CPT5xxbePjfOy8dzQ+hsbOmtWrmNzX/Ic42hpnrDQWGhGxJXAe8Algl4h4GTA1M2+vD1kEnBMRlwD7Am/rqN9EFRpvrp8DuAq4KCIGMtMtSyU1anLfADd/7ezGx9n34ObHGEtNXqdxMXAG8If68UxgRcfzK4BZwDTg0cwcXK/+jNfUzz8KbNVgz5KkUTQSGhFxHPDrzFy63lhrOx5PAtYMU6euDx3TaVLHc5KkljV1eupwYEZE3A1sCWxKFQwzOo6ZDjwIPAxsHhF9mbm6PubB+pjf1Mctj4h+4EXA7xvqWZL0HBqZaWTmAZn515k5G/gn4KuZeSywMiL2qg87ClhSr0/cwroF7qOBJfXXN9aPqZ+/xfUMSeqetq/TmAt8ISI2A+4ELqzrJwFXRMR84FfAEXX9TGBRRPwU+GP9eklSlzQeGpm5iOoTUWTmPcAewxyzDJgzTP0R4K2NNihJKuYut5KkYoaGJKmYoSFJKmZoSJKKGRpSDxhc1d4nydscS+PPhNoaXepV/QMDLPjo+1oZ64P/fHEr42h8cqYhSSpmaEiSihkakqRihoYkqZihIUkqZmhIkooZGpKkYoaGJKmYoSFJKmZoSJKKGRqSpGKGhiSpmKEhSSpmaEiSihkakqRihoYkqZihIUkqZmhIkooZGpKkYoaGJKmYoSFJKmZoSJKKGRqSpGKGhiSpmKEhSSpmaEiSihkakqRihoYkqZihIUkq1t/kbx4R5wLvANYCl2bmgojYH1gATAWuycz59bGzgUuAzYCbgRMzczAitgUWA1sDCczNzMeb7FuSNLzGZhoRsR/wemBnYDfgAxGxC3AZcAjwSmD3iDiofsli4OTM3BGYBBxf1xcCCzNzJ+BHwJlN9SxJGl1joZGZNwGvy8xBqllCP7AF8EBm/qKuLwYOi4iXAVMz8/b65Yvq+gCwL3BtZ72pniVJo2t0TSMzV0XEOcB9wFJgJrCi45AVwKxR6tOAR+uA6axLkrqg8YXwzDwL2ArYBtiRan1jyCRgTd1HSZ26LknqgibXNHaqF7fJzCeBLwNzgBkdh00HHgSWj1B/GNg8Ivrq+oy6LknqgiZnGtsDX4iIF0TEFKrF74uBiIgd6iA4EliSmcuAlRGxV/3ao+r6KuAW4PC6fjSwpMGeJUmjaHIh/EbgBuAu4MfAbZl5NTAPuI5qneN+1i1yzwUuiIj7gU2BC+v6ScAJEXEfsA8wv6meJUmja/Q6jcw8Gzh7vdpSYJdhjr0H2GOY+jKq01qSpC7zinBJUjFDQ5JUzNCQJBUzNCRJxQwNSVIxQ0OSVKwoNCLipcPUXjX27UiSetmo12lExJb1lzdGxByqPaEABqi2BdmpudYkSb3muS7uuwo4oP769x31QdZdyS1JmiBGDY3MPBAgIi7LzPe005IkqVcVbSOSme+pb5S0JetOUZGZdzbVmCSp9xSFRn0jpQ9TbVU+dH+LtVQ72UqSJojSDQuPBnbITO9lIUkTWOl1Gr82MCRJpTONpRFxPvBfwJ+Hiq5pSNLEUhoa8+r/HtZRc01DkiaY0k9Pbdd0I5Kk3lf66akPDlfPzAVj244kqZeVnp76m46vpwD7AUvHvh1JUi8rPT11bOfjiJgJXNpIR5KknrVBW6PXH799+di2IknqdRuypjEJ2I3q6nBJ0gSyIWsaa4FfUW0rIkmaQJ7Xmka9aeFAZv5vo11JknpS6empHaiuBp8JTI6I3wEHZ+bPmmxOktRbShfC/w04PzNfnJmbAx8HLmquLUlSLyoNjZdk5hVDDzLzcmCrZlqSJPWq0tDo77hfOBExjXX31ZAkTRCln576LHB7RFxDFRbvAi5orCtJUk8qnWncSBUWU4BXAS8Frm+qKUlSbyoNjUXARZl5OvBu4AzgsqaakiT1ptLQmJaZFwJk5srM/DQwo7m2JEm96PkshM8cehARL6HaTkSSNIGULoQvAO6OiK9TrW3sj9uISNKEUzTTyMzLqILiLuBHwIGZ+aUmG5Mk9Z7SmQaZeS9w7/P5zSPiLOCd9cMbMvO0iNifauYyFbgmM+fXx84GLgE2A24GTszMwYjYFlgMbA0kMDczH38+fUiSxsYG3U+jRB0ObwR2BWYDr42II6g+dXUI8Epg94g4qH7JYuDkzNyRar3k+Lq+EFiYmTtRzXLObKpnSdLoGgsNYAVwamY+nZmrgJ8BOwIPZOYvMnOQKigOq3fPnZqZt9evXVTXB4B9gWs76w32LEkaRfHpqecrM3869HVEvILqNNVnqcJkyApgFtXuucPVpwGP1gHTWZckdUGTMw0AIuLVwLeoPm31c565Z9UkYE3dR0mdui5J6oJGQyMi9gKWAh+pd8ldzjMvCpwOPDhK/WFg84joq+sz6rokqQuaXAjfBvgKcGRmXl2X76ieih3qIDgSWJKZy4CVdcgAHFXXVwG3AIfX9aOBJU31LEkaXWNrGsCHgBcCCyJiqPZ5YB5wXf3cjaxb5J4LfCEiNgPuBC6s6ycBV0TEfKp7kx/RYM+SpFE0uRB+CnDKCE/vMszx9wB7DFNfBswZ0+YkSRuk8YVwSdL4YWhIkooZGpKkYoaGJKmYoSFJKmZoSJKKGRqSpGKGhiSpmKEhSSpmaEiSihkakqRihoYkqZihIUkqZmhIkooZGpKkYoaGJKmYoSFJKmZoSJKKGRqSpGKGhiSpmKEhSSpmaEiSihkakqRihoYkqZihIUkqZmhIkooZGpKkYoaGJKmYoSFJKmZoSJKKGRqSpGKGhiSpmKEhSSpmaEiSihkakqRi/U0PEBGbAbcBB2fmLyNif2ABMBW4JjPn18fNBi4BNgNuBk7MzMGI2BZYDGwNJDA3Mx9vum9J0rM1OtOIiD2BW4Ed68dTgcuAQ4BXArtHxEH14YuBkzNzR2AScHxdXwgszMydgB8BZzbZsyRpZE2fnjoeeD/wYP14D+CBzPxFZg5SBcVhEfEyYGpm3l4ft6iuDwD7Atd21hvuWZI0gkZPT2XmcQARMVSaCazoOGQFMGuU+jTg0TpgOuuSpC5oeyF8MrC24/EkYM3zqFPXJUld0HZoLAdmdDyeTnXqaqT6w8DmEdFX12ew7lSXJKllbYfGHUBExA51EBwJLMnMZcDKiNirPu6our4KuAU4vK4fDSxpuWdJUq3V0MjMlcA84DrgPuB+1i1yzwUuiIj7gU2BC+v6ScAJEXEfsA8wv82eJUnrNH6dBkBmvrzj66XALsMccw/Vp6vWry8D5jTYniSpkFeES5KKGRqSpGKGhiSpmKEhSSpmaEiSihkakqRihoYkqZihIUkqZmhIkooZGpKkYoaGJKmYoSFJKmZoSJKKGRqSpGKGhiSpmKEhSSpmaEiSihkakqRihoYkqZihIUkqZmhIkooZGpKkYoaGJKmYoSFJKmZoSJKKGRqSpGKGhiSpmKEhSSpmaEiSihkakqRihoYkqZihIUkqZmhIkooZGpKkYoaGJKmYoSFJKtbf7QZKRMSRwHxgAPh0Zl7U5ZYkaULq+ZlGRLwUOA/YG5gNnBARr+puV5I0MW0MM439ge9k5iMAEXEt8A7g3Od4XR/AQw899IziU0/+sYEWn2n58uWjPv/bx1Y23kNJHyv/+GTXe3jkqd74s3j8iT90vYfHnvhz4z2U9PHwo7/reg+PPfZY4z2U9PG7Rx7veg9PPtL830dnHx0/M/uGO27S2rVrW2loQ0XER4FNMnN+/fg4YI/MPOE5Xrc3cEsLLUrSeLRPZt66fnFjmGlMBjqTbRKwpuB1PwT2AVYAqxvoS5LGoz5gBtXP0GfZGEJjOdUP/yHTgQef60WZ+RTwrJSUJD2n/xvpiY0hNL4NnB0RWwFPAP8AjHpqSpLUjJ7/9FRm/gY4A/gucDfwpcz8QXe7kqSJqecXwiVJvaPnZxqSpN5haEiSihkakqRihoYkqdjG8JHbrumVjRIjYjPgNuDgzPxlF8Y/C3hn/fCGzDyt7R7qPs6l2kJmLXBpZi7oRh91L58EpmXmvC6N/11ga2BVXXpfZt7Rcg9vAc4CNgG+mZmntDl+3cNxwMkdpe2Af8/Mk0d4SVN9vBv4aP1wSWZ+qM3xO/r4CHAs8BRwTWaeN9ZjONMYQa9slBgRe1JdpLhj22PX4+8PvBHYlerP4bURcWgX+tgPeD2wM7Ab8IGIiLb7qHt5A3BMN8aux59E9f2wS2bOrn+1HRjbA58H3kb1d/KaiDiozR4AMvOSoT8DYC7wMHB2mz1ExF8BFwL7AbsA+9T/blpVj3kksDvVv9c9I+LtYz2OoTGyv2yUmJlPAEMbJbbteOD9FFwF35AVwKmZ+XRmrgJ+BmzbdhOZeRPwuswcpHqH3U91sWerImJLqjcTn2h77M426v9+MyLuiYhW31XXDqV6J7u8/r44HGg1uIbxOeBjmdnODn/r9FH9LN2E6qzEANDO7pPPtCvwjcx8NDNXA1+nCvUxZWiMbCbVD8whK4BZbTeRmcdlZtc2XszMn2bm7QAR8Qqq01Q3dqmXVRFxDnAfsBT4TRfauJjqYtPmt8Ud2Yup/v8PBd4AnBgRB7Tcww5AX0R8NSLuBk6ii38m9bvsqZn5n22PnZmPAWcC91Nte/RLqtPJbbsTODAitoyIFwJvpdp2aUwZGiPb0I0Sx6WIeDXwLeDDmflAt/rIzLOArYBtqGZhranPn/86M5e2Oe76MvP7mXl0Zv6pfld9KfCmltvop5qNvxf4O2BPunjKDngf0JU1rojYGXgP8DKqN5urgdbXNOrvy0XA96hmGbcCT4/1OIbGyJZT7fQ4pGijxPEoIvaiemf7kcy8oks97BQRswEy80ngy1Tn0tt0OPDG+p31ucBbI+KClnsgIvau11WGTGLdgnhbHgK+nZm/zcw/A9cDe7TcAwARMYVqPeGr3RgfOBBYmpkP1xulLgLmtN1ERLwIuC4zd87MOVSL4SNuPLih/PTUyNwoEYiIbYCvAIdn5ne62Mr2wDn1fVLWAocAl7XZQGb+5RRQRMwD5mTmP7bZQ20L4NyI+Huq8+fHACe23MPXgCsiYgvgMeAgqu+TbtgZ+J967bEb7gHOj4hNgCeBtzDCtuIN2w74YkTsRrW+8t7615hypjECN0r8iw8BLwQWRMTd9a+2f0CRmTcCNwB3AT8GbsvMq9vuoxdk5td45p/FZZn5/ZZ7uAM4n+oUyH3AMuDyNnvosD3VmYGuyMxvAldR/V3cSxXk/9KFPu4Frqt7+AHVZQL/PdbjuGGhJKmYMw1JUjFDQ5JUzNCQJBUzNCRJxQwNSVIxQ0MaIxExJyJ+8hzHrI2Iac/z910UEV3ZNVVan6EhSSrmFeHSGIuIHYGLgBdRbUVzN9UV9SvrQ86LiN2p3rTNry/WIyLeS7Xx32Tg98DJmXl/2/1Lo3GmIY2944ErMvNvqXaD3Q54c8fzP8/M1wDvptqKY6v6fiHHAPtk5q5UV1tf33Lf0nNypiGNvdOBAyLiNKqbJc0ENu14/vMAmfmTiLiPapfYvakC5raOe0u9uL5/h9QzDA1p7F1F9W/rP6j2iNqWaifaIas7vp5MtUNtH9VtSk8HiIjJVGHTzft2SM/i6Slp7B0InJuZ19SP96QKhSHzACLiNVSzizuAbwBHRMTQdvwnUm1HL/UUZxrS2PsYcH1EPAH8CbiJKhyGbB8Rd1Ft8f6uzHyE6tat/wp8KyLWAI8Cb8/MtV26Fbo0LHe5lSQV8/SUJKmYoSFJKmZoSJKKGRqSpGKGhiSpmKEhSSpmaEiSihkakqRi/w//J2Ur9cBB5wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "Y_train = train['label']\n",
    "X_train = train.drop(labels=['label'], axis=1)\n",
    "g = sns.countplot(Y_train)\n",
    "X_train.shape, Y_train.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.2 Check for null and missing values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count       784\n",
       "unique        1\n",
       "top       False\n",
       "freq        784\n",
       "dtype: object"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.isnull().any().describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count       784\n",
       "unique        1\n",
       "top       False\n",
       "freq        784\n",
       "dtype: object"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.isnull().any().describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.3 Normalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = X_train / 255.0\n",
    "test = test / 255.0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.4 Reshape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = X_train.values.reshape(-1, 28, 28, 1)\n",
    "test = test.values.reshape(-1, 28, 28, 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.5 Label encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_train = to_categorical(Y_train, num_classes=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.6 Split training and valdiation set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((34020, 28, 28, 1), (3780, 28, 28, 1), (34020, 10), (3780, 10))"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size=0.1, random_state=2)\n",
    "X_train.shape, X_val.shape, Y_train.shape, Y_val.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. CNN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.1 Define the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From D:\\Programs\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:74: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
      "\n",
      "WARNING:tensorflow:From D:\\Programs\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:517: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
      "\n",
      "WARNING:tensorflow:From D:\\Programs\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:4138: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
      "\n",
      "WARNING:tensorflow:From D:\\Programs\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:3976: The name tf.nn.max_pool is deprecated. Please use tf.nn.max_pool2d instead.\n",
      "\n",
      "WARNING:tensorflow:From D:\\Programs\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:133: The name tf.placeholder_with_default is deprecated. Please use tf.compat.v1.placeholder_with_default instead.\n",
      "\n",
      "WARNING:tensorflow:From D:\\Programs\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "\n",
    "model.add(Conv2D(filters=32, kernel_size=(5, 5), padding='Same', activation='relu', input_shape=(28, 28, 1)))\n",
    "model.add(Conv2D(filters=32, kernel_size=(5, 5), padding='Same', activation='relu'))\n",
    "model.add(MaxPool2D(pool_size=(2, 2)))\n",
    "model.add(Dropout(0.25))\n",
    "\n",
    "model.add(Conv2D(filters=64, kernel_size=(3, 3), padding='Same', activation='relu'))\n",
    "model.add(Conv2D(filters=64, kernel_size=(3, 3), padding='Same', activation='relu'))\n",
    "model.add(MaxPool2D(pool_size=(2, 2), strides=(2, 2)))\n",
    "model.add(Dropout(0.25))\n",
    "\n",
    "model.add(Flatten())\n",
    "model.add(Dense(256, activation='relu'))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(10, activation='softmax'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.2 Set the optimizer and annealer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From D:\\Programs\\Anaconda3\\lib\\site-packages\\keras\\optimizers.py:790: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n",
      "WARNING:tensorflow:From D:\\Programs\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:3295: The name tf.log is deprecated. Please use tf.math.log instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc', patience=3, verbose=1, factor=0.5, min_lr=0.00001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs = 30\n",
    "batch_size=86"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.3 Data augmentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "datagen = ImageDataGenerator(featurewise_center=False,             # set input mean to 0 over the dataset\n",
    "                             samplewise_center=False,              # set each sample mean to 0\n",
    "                             featurewise_std_normalization=False,  # divide inputs by std of the dataset\n",
    "                             samplewise_std_normalization=False,   # divide each input by its std\n",
    "                             zca_whitening=False,                  # apply ZCA whitening\n",
    "                             rotation_range=10,                    # randomly rotate images in the range (degrees, 0 to 180)\n",
    "                             zoom_range = 0.1,                     # Randomly zoom image \n",
    "                             width_shift_range=0.1,                # randomly shift images horizontally (fraction of total width)\n",
    "                             height_shift_range=0.1,               # randomly shift images vertically (fraction of total height)\n",
    "                             horizontal_flip=False,                # randomly flip images\n",
    "                             vertical_flip=False                   # randomly flip images\n",
    "                            )\n",
    "datagen.fit(X_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For the data augmentation, i choosed to :\n",
    "\n",
    "- Randomly rotate some training images by 10 degrees\n",
    "- Randomly Zoom by 10% some training images\n",
    "- Randomly shift images horizontally by 10% of the width\n",
    "- Randomly shift images vertically by 10% of the height"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From D:\\Programs\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\math_grad.py:1250: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "Epoch 1/30\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-22-e7250a67be0b>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      2\u001b[0m                              \u001b[0mepochs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mepochs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalidation_data\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_val\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mY_val\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m                              \u001b[0mverbose\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msteps_per_epoch\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mX_train\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m//\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 4\u001b[1;33m                              callbacks=[learning_rate_reduction])\n\u001b[0m",
      "\u001b[1;32mD:\\Programs\\Anaconda3\\lib\\site-packages\\keras\\legacy\\interfaces.py\u001b[0m in \u001b[0;36mwrapper\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     89\u001b[0m                 warnings.warn('Update your `' + object_name + '` call to the ' +\n\u001b[0;32m     90\u001b[0m                               'Keras 2 API: ' + signature, stacklevel=2)\n\u001b[1;32m---> 91\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     92\u001b[0m         \u001b[0mwrapper\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_original_function\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     93\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Programs\\Anaconda3\\lib\\site-packages\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mfit_generator\u001b[1;34m(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[0;32m   1416\u001b[0m             \u001b[0muse_multiprocessing\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0muse_multiprocessing\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1417\u001b[0m             \u001b[0mshuffle\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1418\u001b[1;33m             initial_epoch=initial_epoch)\n\u001b[0m\u001b[0;32m   1419\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1420\u001b[0m     \u001b[1;33m@\u001b[0m\u001b[0minterfaces\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlegacy_generator_methods_support\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Programs\\Anaconda3\\lib\\site-packages\\keras\\engine\\training_generator.py\u001b[0m in \u001b[0;36mfit_generator\u001b[1;34m(model, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[0;32m    215\u001b[0m                 outs = model.train_on_batch(x, y,\n\u001b[0;32m    216\u001b[0m                                             \u001b[0msample_weight\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 217\u001b[1;33m                                             class_weight=class_weight)\n\u001b[0m\u001b[0;32m    218\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    219\u001b[0m                 \u001b[0mouts\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mto_list\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Programs\\Anaconda3\\lib\\site-packages\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mtrain_on_batch\u001b[1;34m(self, x, y, sample_weight, class_weight)\u001b[0m\n\u001b[0;32m   1215\u001b[0m             \u001b[0mins\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mx\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0my\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0msample_weights\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1216\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_make_train_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1217\u001b[1;33m         \u001b[0moutputs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mins\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1218\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0munpack_singleton\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1219\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Programs\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, inputs)\u001b[0m\n\u001b[0;32m   2713\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_legacy_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2714\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2715\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2716\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2717\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mpy_any\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mis_tensor\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[1;32min\u001b[0m \u001b[0minputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Programs\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py\u001b[0m in \u001b[0;36m_call\u001b[1;34m(self, inputs)\u001b[0m\n\u001b[0;32m   2673\u001b[0m             \u001b[0mfetched\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun_metadata\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2674\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2675\u001b[1;33m             \u001b[0mfetched\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2676\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mfetched\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2677\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Programs\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1456\u001b[0m         ret = tf_session.TF_SessionRunCallable(self._session._session,\n\u001b[0;32m   1457\u001b[0m                                                \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_handle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1458\u001b[1;33m                                                run_metadata_ptr)\n\u001b[0m\u001b[0;32m   1459\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1460\u001b[0m           \u001b[0mproto_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "history = model.fit_generator(datagen.flow(X_train, Y_train, batch_size=batch_size),\n",
    "                             epochs=epochs, validation_data=(X_val, Y_val),\n",
    "                             verbose=2, steps_per_epoch=X_train.shape[0] // batch_size,\n",
    "                             callbacks=[learning_rate_reduction])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
